'''
Created on 2015-3-16

@author: xizhzhao
'''
import urllib, urllib2, urlparse, cookielib
import os
from Configurator import Configurator
from threadWorker import ThreadPool
from Logger import * 
import Queue
from urlArrays import *

# seen_urls = []
# urls_to_visit = []

class DataFetcher(object):
    def init(self, fetch_url, recurse=False):
        self.__url__ = None
        self.__mutex__ = threading.Lock()

        self.__url__ = fetch_url
        if self.__url__ == None: return False
        
        self.__data_src__ = None
        
        self.__page_uuid__ = None  # uid of this page
        self.__next_links__ = []  # containing links in this page
        
        self.__page_md5__ = None  # Md5 examing code for this page
        self.__time__ = None  # First fetch time
        self.__validate__ = lambda url: True
        
        self.__proxy_needed__ = (Configurator.get_val('proxy_enable') == 'true') 
        self.__cookie_needed__ = (Configurator.get_val('cookie_need') == 'true')
        self.__recurse__ = recurse
        
        self.__save_path__ = Configurator.get_val('page_lib')
        return True

    def set_recurse(self, recurse):
        self.__recurse__ = recurse

    # used to customize the fecth process
    def preFectch(self, params, headers): return

    def fetch(self):
        global logger

        params = {}
        try:
            if self.__proxy_needed__:
                proxy_handler = urllib2.ProxyHandler(
                    {
                       Configurator.get_val('proxy_type') : Configurator.get_val('proxy_url')
                    }
                )
                opener = urllib2.build_opener(proxy_handler,
                                              urllib2.HTTPHandler)
                urllib2.install_opener(opener)

            if self.__cookie_needed__:
                cookie = cookielib.CookieJar()
                import PageFetcher
                cookie_handler = PageFetcher.NoExceptionCookieProcesser(cookie)
                opener = urllib2.build_opener(cookie_handler, urllib2.HTTPHandler)
                urllib2.install_opener(opener)
                params = {"username": Configurator.get_val('username'),
                          "password": Configurator.get_val('password')}

        except urllib2.URLError, e:
            logger.error(e)
            logger.error("Fetch Failed!")
            return False

        '''
            Code here comes from Page http://blog.csdn.net/wklken/article/details/7364390
        '''
        logger.info("Fetching url: %s" % (self.__url__))
        headers = {'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6'}
        self.preFectch(params, headers)
        data_src = self.__fetch__(self.__url__, params, headers)

        # import gzip
        # import StringIO
        # data = StringIO.StringIO(data_src)
        # gzipper = gzip.GzipFile(fileobj=data)
        # data_src = gzipper.read()

        # TODO:: check the encoding logic
        import chardet
        encoding = chardet.detect(data_src)['encoding']
        if encoding == None or encoding == 'None' or ''.join(encoding.split()) == '':
             encoding = 'UTF8'

        '''
            Here we should pass unicode to html_src
        '''
        self.__data_src__ = data_src.decode(encoding, 'ignore')
        self.__encoding__ = encoding # all convert to UTF8

        logger.info("Fetch Success!")
        return True

    def __fetch__(self, url, params, headers):
        try:
            # add this to pretend to be not a spider

            req = urllib2.Request(url=url, headers=headers)
            if params == None or len(params) > 0:
                reader = urllib2.urlopen(req, urllib.urlencode(params))
            else:
                reader = urllib2.urlopen(req)

        except urllib2.HTTPError as e:
            logger.error(e)
            return False
        except ValueError as e:
            logger.error(e)
            return False

        return reader.read()

    def analysis(self): return
    
    def add_validation_rule(self, validate):
        return  # user defined validation method    
